# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

from components.component.scrapy_redis_bloomfilter_custom.bloomfilter import BloomFilter
from components.component.scrapy_redis_custom import get_redis_from_settings


# useful for handling different item types with a single interface


class TaxPolicyDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        cls.server = get_redis_from_settings(crawler.spider.settings)
        if crawler.spider.settings.getbool("BATCH_RUN_ENABLE", False):
            bf = BloomFilter(get_redis_from_settings(crawler.spider.settings), crawler.spider.settings.get("BATCH_RUN_REDIS_KEY", "tax_policy_url_bloom_filter"), 30, 6)
            setattr(crawler.spider, 'bf', bf)
        return s

    def request_fingerprint(self, request, spider):
        return spider.crawler.request_fingerprinter.fingerprint(request).hex()

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        self.server.sadd('tax_policy_parser_name', request.callback.__name__)
        bf = getattr(spider, 'bf', None)
        if bf:
            batch_run_parser_name_list = spider.settings.getlist("BATCH_RUN_PARSER_NAME_LIST", ['parse_content', 'parse_detail'])
            if request.callback.__name__ in batch_run_parser_name_list:
                fp = self.request_fingerprint(request, spider)
                bf.insert(fp)
                spider.logger.debug(f"bloom filter request url add : {request.url}")
        return response

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)
